﻿using Sgml;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Xml;

namespace NudeTeensArtImgDownloader
{
    class Program
    {

        class FailedInfo
        {
            public string Url;
            public string Message;
        }

        static string[] _subDirNames = new string[] {
            "AmourAngels",
            "Art-Lingerie",
            "AvErotica",
            "Babes",
            "BareMaidens",
            "BeautifulNude",
            "BreathTakers",
            "Breath-Takers",
            "EroticBeauty",
            "EroticSnap",
            "Errotica-Archives",
            "EvasGarden",
            "HayleysSecrets",
            "Hegre-Art",
            "JTS",
            "MCN",
            "MC-Nudes",
            "Met-Art",
            "MetModels",
            "MGS",
            "MPL",
            "NakedBy",
            "Orgasms",
            "P4E",
            "PeterJanhans",
            "PhotoDromm",
            "Pretty4Ever",
            "PrettyNudes",
            "PureBeauty",
            "Querro",
            "Rigin-Studio",
            "SexArt",
            "Sex-Art",
            "SheDontBlush",
            "SinfulGoddess",
            "SinfulGoddesses",
            "Skokoff",
            "Teenrotica",
            "TheLifeErotic",
            "TorridArt",
            "W4B",
            "WalterBosque",
            "Watch4Beauty",
            "X-Art",
            "Zemani",
        };

        static Dictionary<string, string> _subDirReplaceTbl = new Dictionary<string, string>() {
            {"Breath-Takers", "BreathTakers"},
            {"Sex-Art", "SexArt"},
        };

        static Encoding _enc = Encoding.GetEncoding("utf-8");

        static void Main(string[] args) {
            ServicePointManager.DefaultConnectionLimit = 10;
            string outputDir = @"e:\画像\NudeTeensArt";

            try {
                for (int page = 1; ; ++page) {
                    string pageUrl = string.Format("http://setsdb.org/page/{0}/", page);
                    // ページURLからエントリURLを取得
                    List<EntryInfo> entryInfos = getEntryInfo(pageUrl);
                    foreach (EntryInfo entryInfo in entryInfos) {
                        string dirName1 = correctFileName(entryInfo.Title);
                        string entryDir = outputDir;

                        //// 出力先ディレクトリが存在する場合、ダウンロードしない
                        //if (Directory.Exists(entryDir) && !isExistFailedLog(entryDir) && !isCoveredOnly(entryDir))
                        //    continue;

                        string dirName2 = null;
#if true
                        foreach (string subDirName in _subDirNames) {
                            int index = entryInfo.Title.IndexOf(subDirName, StringComparison.OrdinalIgnoreCase);
                            if (0 == index || 1 == index) {
                                if (_subDirReplaceTbl.ContainsKey(subDirName))
                                    dirName2 = _subDirReplaceTbl[subDirName];
                                else
                                    dirName2 = subDirName;
                            }
                        }
#else
                        // フォルダ数が多いので Met-Art などの名前のフォルダを作成する
                        // タイトルの例:
                        // Met-Art – 2003-12-26 – Lucy S – Sunset Goddess
                        // [JTS] – 2005-06-10 – Fatal (x56) 824×1241
                        // MPL – 2005-05-03 – Alexandra – Shoot Day – Behind the Scenes 2
                        // PrettyNudes – 2007-08-xx – Nicol – High Fatra Bonus
                        // PeterJanhans – 2006-03 – Polina – Spread
                        // SexArt 2012-04-13 Elle Alexandra & Lexi Bloom & Malena Morgan – Stepping Out (HD Video)
                        // Art-Lingerie - 2009-05-02 – Renata
                        // SheDontBlush – 2010-08-12  - Denisa – Black n White
                        Regex reg = new Regex(@"(?<dir2>.*?) (–|-)( | )\d\d\d\d-\d\d(-\d\d|-xx)?( | )*(–|-)", RegexOptions.IgnoreCase);
                        Match match = reg.Match(entryInfo.Title);
                        if (match.Success) {
                            dirName2 = match.Groups["dir2"].Value;
                            if (dirName2.StartsWith("[", StringComparison.Ordinal) && dirName2.EndsWith("]", StringComparison.Ordinal))
                                dirName2 = dirName2.Substring(1, dirName2.Length - 2);
                        }
                        // [Errotica-Archives] 2011-01-10 Raisa – Avvincere (x68) 4032×6048
                        if (null == dirName2) {
                            reg = new Regex(@"[(?<dir2>.*?)] \d\d\d\d-\d\d-\d\d ", RegexOptions.IgnoreCase);
                            match = reg.Match(entryInfo.Title);
                            if (match.Success) {
                                dirName2 = match.Groups["dir2"].Value;
                            }
                        }
#endif
                        if (null != dirName2) {
                            dirName2 = correctFileName(dirName2);
                            entryDir = Path.Combine(entryDir, dirName2);
                        }
                        entryDir = Path.Combine(entryDir, dirName1);

                        // 出力先ディレクトリが存在する場合、ダウンロードしない
                        if (Directory.Exists(entryDir) && !isExistFailedLog(entryDir) && !isCoverOnly(entryDir))
                            continue;

                        // エントリURLから画像ページURLを取得
                        Console.WriteLine("●{0} [{1}] から画像ページURLを取得", entryInfo.Url, entryInfo.Title);
                        List<string> imagePageUrls = getImagePageUrl(entryInfo.Url);

                        Directory.CreateDirectory(entryDir);

                        // infoファイル作成
                        createInfoFile(entryInfo, imagePageUrls, entryDir);
                        // エントリページへのショートカットファイルを作成
                        createInternetShortcut(entryInfo.Url, entryDir);
                        // カバー画像しかない場合は _coverOnly.txt を出力
                        string coverOnlyFile = Path.Combine(entryDir, "_coverOnly.txt");
                        File.Delete(coverOnlyFile);
                        if (1 == imagePageUrls.Count)
                            File.Create(coverOnlyFile).Close();

                        List<FailedInfo> failedInfos = new List<FailedInfo>();

                        Console.WriteLine("●{0} の画像をダウンロード({1})", entryInfo.Url, imagePageUrls.Count);
                        Parallel.ForEach(imagePageUrls, new ParallelOptions() { MaxDegreeOfParallelism = 3 }, (imagePageUrl, state, index) => {
                            try {
                                downloadImage(imagePageUrl, (int)index, entryDir);
                                Console.Write("o");
                            }
                            catch (Exception exp) {
                                Console.Write("X");

                                FailedInfo failedInfo = new FailedInfo() { Url = imagePageUrl };

                                if (exp is ImageNotExist) {
                                    failedInfo.Message = "Image File Not Exist";
                                }
                                else {
                                    WebException we = exp as WebException;
                                    if (null == we)
                                        we = exp.InnerException as WebException;
                                    if (null != we)
                                        failedInfo.Message = we.Message;
                                }

                                if (null == failedInfo.Message) {
                                    Console.WriteLine("{0} ({1}) のダウンロードを中止", entryInfo.Url, entryInfo.Title);
                                    throw exp;
                                }

                                lock (failedInfos) {
                                    failedInfos.Add(failedInfo);
                                }
                            }
                        });
                        Console.Write("\n");

                        // ダウンロードに失敗したファイルを failed～.txt に出力
                        if (0 < failedInfos.Count) {
                            // 今回失敗したのが全て存在する場合は削除
                            List<string> deleteLogFiles = new List<string>();
                            string[] logFiles = Directory.GetFiles(entryDir, "_failed_*.txt");
                            foreach (string logFile in logFiles) {
                                List<string> urls = new List<string>();
                                using (StreamReader sr = new StreamReader(logFile, _enc)) {
                                    string line;
                                    while ((line = sr.ReadLine()) != null) {
                                        string[] parts = line.Split('\t');
                                        urls.Add(parts[0]);
                                    }
                                }
                                bool isDelete = true;
                                foreach (FailedInfo failedInfo in failedInfos) {
                                    if (!urls.Contains(failedInfo.Url)) {
                                        isDelete = false;
                                        break;
                                    }
                                }
                                if (isDelete)
                                    deleteLogFiles.Add(logFile);
                            }

                            // 新規作成
                            DateTime now = DateTime.Now;
                            string failedFilePath = Path.Combine(entryDir, string.Format("_failed_{0}.txt", now.ToString("yyMMdd_HHmmss")));
                            using (StreamWriter writer = new StreamWriter(failedFilePath, true, _enc)) {
                                foreach (FailedInfo failedInfo in failedInfos) {
                                    writer.WriteLine(failedInfo.Url + "\t" + failedInfo.Message);
                                }
                            }
                            // 削除
                            foreach (string deleteLogFile in deleteLogFiles)
                                File.Delete(deleteLogFile);
                        }
                        else {
                            // 失敗したファイルがない場合、_faild_～.txt を削除
                            string[] logFiles = Directory.GetFiles(entryDir, "_failed_*.txt");
                            foreach (string logFile in logFiles)
                                File.Delete(logFile);
                        }
                    }
                }
            }
            catch (Exception exp) {
                Console.WriteLine(exp.ToString());
            }
            Console.WriteLine("終了");
            Console.ReadKey();
        }

        static bool isCoverOnly(string dir) {
            int lineCount = 0;
            try {
                using (StreamReader sr = new StreamReader(Path.Combine(dir, "_info.txt"), _enc)) {
                    string line;
                    while ((line = sr.ReadLine()) != null) {
                        ++lineCount;
                    }
                }
            }
            catch (Exception exp) {
                return true;   // _info.txtが存在しない場合、再チェック
            }
            return 4 == lineCount;
        }

        // _failed_～.txt が存在するかどうか
        static bool isExistFailedLog(string dir) {
            string[] ret = Directory.GetFiles(dir, "_failed_*.txt");
            return ret.Length > 0;
        }

        static string correctFileName(string src) {
            src = src.Trim();
            char[] invalidChar = Path.GetInvalidFileNameChars();
            foreach (char c in invalidChar) {
                src = src.Replace(c, '_');
            }
            return src;
        }

        static void createInfoFile(EntryInfo entryInfo, List<string> imagePageUrls, string entryDir) {
            string infoFilePath = Path.Combine(entryDir, "_info.txt");
            using (StreamWriter infoWriter = new StreamWriter(infoFilePath, false, _enc)) {
                infoWriter.WriteLine(entryInfo.Url);
                infoWriter.WriteLine(entryInfo.Title);
                infoWriter.WriteLine("");
                foreach (string imagePageUrl in imagePageUrls) {
                    infoWriter.WriteLine(imagePageUrl);
                }
            }
        }

        static void createInternetShortcut(string targetUrl, string entryDir) {
            string linkFilePath = Path.Combine(entryDir, "_entry.url");
            using (StreamWriter sw = new StreamWriter(linkFilePath, false, _enc)) {
                sw.WriteLine("[InternetShortcut]");
                sw.WriteLine("URL=" + targetUrl);
            }
        }

        // url = http://setsdb.org/lana-lanky/
        // ※ ret[0] はカバー画像のURLで画像ページURLではない
        static List<string> getImagePageUrl(string url) {
            WebResponse res = getWebResponse(url);

            string src;

            using (Stream st = res.GetResponseStream())
            using (StreamReader sr = new StreamReader(st, _enc)) {
                src = sr.ReadToEnd();
            }

            XmlDocument xmlDoc = new XmlDocument();
            using (SgmlReader sgmlReader = new SgmlReader()) {
                sgmlReader.DocType = "HTML";
                sgmlReader.CaseFolding = CaseFolding.ToLower;
                sgmlReader.InputStream = new StringReader(src);
                xmlDoc.Load(sgmlReader);
            }

            //xmlDoc.Save("test.xml");

            List<string> ret = new List<string>();

            XmlNamespaceManager nsm = new XmlNamespaceManager(xmlDoc.NameTable);
            nsm.AddNamespace("x", "http://www.w3.org/1999/xhtml");

            // タイトル画像
            // xpath: //div[@class='entry']/p[2]/img
            //        //div[@class='entry']/p[2]/strong/img 
            // <img src="http://img11.nnm2.com/5/b/3/f/e/95fa973505fa7aea4dc0dd2a67b.jpg">
            XmlNode node = xmlDoc.SelectSingleNode("//x:div[@class='entry']/x:p[2]//x:img", nsm);
            ret.Add(node.Attributes["src"].Value);
            
            // 画像ページ
            // xpath: //div[@class='entry']/p[5]/a[1～]
            // <a target="_blank" href="http://imgchili.com/show/21186/21186379_metart_sperto_kather.jpg">
            // ※divの5番目の子供がdivでその中にaが入っている場合があるので node[5] で取得
            XmlNodeList nodes = xmlDoc.SelectNodes("//x:div[@class='entry']/*[5]/x:a", nsm);
            foreach (XmlNode node2 in nodes) {
                string imagePageUrl = node2.Attributes["href"].Value;
                ret.Add(imagePageUrl);
            }

            return ret;
        }

        class ImageNotExist : Exception
        {
        }

        class UnknownHost : Exception
        {
            public UnknownHost(string url)
                : base(url) {
            }
        }

        static string getImageUrlFromImagePage(string imagePageUrl) {
            //Console.WriteLine("★{0} から画像URLを取得", imagePageUrl);

            WebResponse res = getWebResponse(imagePageUrl);

            string src;

            using (Stream st = res.GetResponseStream())
            using (StreamReader sr = new StreamReader(st, _enc)) {
                src = sr.ReadToEnd();
            }

            XmlDocument xmlDoc = new XmlDocument();
            using (SgmlReader sgmlReader = new SgmlReader()) {
                sgmlReader.DocType = "HTML";
                sgmlReader.CaseFolding = CaseFolding.ToLower;
                sgmlReader.InputStream = new StringReader(src);
                xmlDoc.Load(sgmlReader);
            }

            XmlNamespaceManager nsm = new XmlNamespaceManager(xmlDoc.NameTable);
            nsm.AddNamespace("x", "http://www.w3.org/1999/xhtml");

            Uri uri = new Uri(imagePageUrl);
            if (uri.Host == "sharenxs.com") {
                // xpath: //*[@id="img1"]
                // <img id="img1" src="http://cache.sharenxs.com/images/wz/cab-/ce/ag/bg/b-/001._cmdqr27.jpg" style="width: 600px;">
                XmlNode node = xmlDoc.SelectSingleNode("//img[@id='img1']");
                return node.Attributes["src"].Value;
            }
            else if (uri.Host == "imgchili.com") {
                // xpath: //*[@id="show_image"]
                // <img width="908" height="1363" alt="21186378_metart_sperto_kather.jpg" src="http://i4.imgchili.net/21186/21186378_metart_sperto_kather.jpg" onclick="scale(this);" onload="scale(this);" id="show_image" style="cursor: url(&quot;/theme/magnify.cur&quot;), default;">
                XmlNode node = xmlDoc.SelectSingleNode("//x:img[@id='show_image']", nsm);
                if (null == node) {
                    //<div class="message_box">
                    //    The image file <b>18440887_eb_sunshine_fun_mile.jpg</b> does not exist. <br/>
                    //    Please ensure the filename is spelled correctly.<br/>
                    //    <br/>
                    //</div>
                    // のような内容がある場合がある。
                    node = xmlDoc.SelectSingleNode("//x:div[@class='message_box']", nsm);
                    if (0 <= node.InnerText.IndexOf("does not exist", StringComparison.OrdinalIgnoreCase)) {
                        throw new ImageNotExist();
                    }
                    throw new Exception(uri.AbsoluteUri + " から画像URLを取得できませんでした。");
                }
                  return node.Attributes["src"].Value;
            }
            else if (uri.Host == "www.turboimagehost.com") {
                // xpath: //*[@id="imageid"]
                // <img width="900" height="1354" border="0" onload="scaleImg(); load_ads(); if(!aweGetCookie('entercook')){popup('popUpDiv1');}" alt="sonia 601" id="imageid" class="upimage" src="http://s4d3.turboimagehost.com/sp/28574144f8569aea4fe5f0ece57caef3/sonia601.jpg">
                XmlNode node = xmlDoc.SelectSingleNode("//x:img[@id='imageid']", nsm);
                return node.Attributes["src"].Value;
            }
            else if (uri.Host == "www.pixhost.org") {
                // xpath: //*[@id="show_image"]
                // <img width="1096" height="822" border="0" alt="17479682_yannasexyshower_032707_001xxxl.jpg" onload="scaleImg(100)" onclick="showOnclick(100)" src="http://img3.pixhost.org/images/3450/17479682_yannasexyshower_032707_001xxxl.jpg" id="show_image" style="cursor: pointer;">
                XmlNode node = xmlDoc.SelectSingleNode("//x:img[@id='show_image']", nsm);
                return node.Attributes["src"].Value;
            }
            else if (uri.Host.EndsWith(".imagevenue.com")) {
                // xpath: //*[@id="thepic"]
                // <img alt="aAfkjfp01fo1i-23835/loc401/65107_pj03katya01of42_1600_122_401lo.jpg" src="aAfkjfp01fo1i-23835/loc401/65107_pj03katya01of42_1600_122_401lo.jpg" onload="scaleImg();" id="thepic">
                XmlNode node = xmlDoc.SelectSingleNode("//img[@id='thepic']");
                if (null == node) {
                    node = xmlDoc.SelectSingleNode("//body/text()");
                    if (null == node) {
                        // http://img242.imagevenue.com/img.php?image=07910_MetArt_Likkix_Karina_B_by_Voronin_high_0127_122_185lo.jpg
                        // の内容が以下のような途中で終わってる不正なHTMLだったので、それのチェック
                        //<HTML>
                        //<BODY>
                        //<STYLE type="text/css"></STYLE>
                        //<script language=Javascript></script>
                        src = src.TrimEnd();
                        if (src.EndsWith("</script>")) {
                            throw new ImageNotExist();
                        }
                        Console.WriteLine("\"" + src + "\"");
                    }
                    else if (0 <= node.Value.IndexOf("This image does not exist on this server", StringComparison.OrdinalIgnoreCase)) {
                        throw new ImageNotExist();
                    }
                    throw new Exception(uri.AbsoluteUri + " から画像URLを取得できませんでした。");
                }
                return new Uri(uri, node.Attributes["src"].Value).AbsoluteUri;
            }
            else if (uri.Host == "www.imagebam.com") {
                // xpath: //div[@id='imageContainer']/table/tr[2]/td/img
                XmlNode node = xmlDoc.SelectSingleNode("//x:div[@id='imageContainer']/x:table/x:tr[2]/x:td/x:img", nsm);
                return node.Attributes["src"].Value;
            }
            else if (uri.Host == "imagetwist.com") {
                XmlNode node = xmlDoc.SelectSingleNode("//x:img[@class='pic']", nsm);
                return node.Attributes["src"].Value;
            }

            throw new UnknownHost("★unknown host: " + uri.Host);
        }

        class EntryInfo
        {
            public string Url;      // http://setsdb.org/lana-lanky/#more-27318
            public string Title;    // Hegre-Art – 2007-01-13 – Lana – Lanky
        }

        // url = http://setsdb.org/gallery/page/2/
        static List<EntryInfo> getEntryInfo(string url) {
            Console.WriteLine("★{0} からエントリURLを取得", url);

            WebResponse res = getWebResponse(url);

            string src;

            using (Stream st = res.GetResponseStream())
            using (StreamReader sr = new StreamReader(st, _enc)) {
                src = sr.ReadToEnd();
            }

            XmlDocument xmlDoc = new XmlDocument();
            using (SgmlReader sgmlReader = new SgmlReader()) {
                sgmlReader.DocType = "HTML";
                sgmlReader.InputStream = new StringReader(src);
                xmlDoc.Load(sgmlReader);
            }

            //xmlDoc.Save("test.xml");

            List<EntryInfo> ret = new List<EntryInfo>();

            // entryのURL
            // xpath: //div[@class='entry']/p[4]/a
            // <a class="more-link" href="http://setsdb.org/jana-corridor/#more-25650">View Image Gallery / Video Clip</a>
            XmlNamespaceManager nsm = new XmlNamespaceManager(xmlDoc.NameTable);
            nsm.AddNamespace("x", "http://www.w3.org/1999/xhtml");
            XmlNodeList nodes = xmlDoc.SelectNodes(@"//x:div[@class='entry']/x:p[4]/x:a", nsm);
            foreach (XmlNode node in nodes) {
                EntryInfo entryInfo = new EntryInfo();
                ret.Add(entryInfo);

                entryInfo.Url = node.Attributes["href"].Value;
                // タイトル
                // xpath: //div[@class='entry']/p/strong
                // <strong>NakedBy &ndash; 2008-03-27 &ndash; Jana &ndash; Corridor</strong>
                //XmlNode titleNode = node.SelectSingleNode("../../x:p/x:strong", nsm);

                // "Babes – 2012-07-02 – Dani Jensen – Naughty (HD Video)"のエントリのタイトルが<strong>になっていなかったので
                // strongの外側のInnerTextを取得するように変更。
                XmlNode titleNode = node.SelectSingleNode("../../x:p", nsm);
                entryInfo.Title = titleNode.InnerText;
            }

            return ret;
        }

        static void downloadImage(string imagePageUrl, int index, string dir) {
            string url = null;
            if (0 != index) {
                int retry = 0;
                do {
                    try {
                        url = getImageUrlFromImagePage(imagePageUrl);
                    }
                    catch (UnknownHost exp) {
                        throw exp;
                    }
                    catch (ImageNotExist exp) {
                        throw exp;
                    }
                    catch (WebException exp) {
                        throw exp;
                    }
                    catch (Exception exp) {
                        // 取得したhtmlが不正でNREが発生することがあって、リトライすると正しいhtmlが取得できることがあったので。
                        ++retry;
                        if (retry > 10)
                            throw new Exception(imagePageUrl + " から画像URLを取得できませんでした。", exp);
                        Thread.Sleep(TimeSpan.FromSeconds(10));
                    }
                } while (null == url);
            }
            else
                url = imagePageUrl;

            // 保存ファイルパスを作成
            string filePath;
            if (0 == index) {   // 0 はタイトル画像でランダムな名前になっているようなので、
                string ext = Path.GetExtension(url);
                if (string.IsNullOrEmpty(ext))
                    throw new Exception("urlから拡張子を取得するのに失敗");

                filePath = Path.Combine(dir, "_cover" + ext);
            }
            else {
                filePath = Path.Combine(dir, Path.GetFileName(url));
            }

            // ファイルが存在している場合、ダウンロードしない
            if (File.Exists(filePath))
                return;

            // WebRequestの作成
            //サーバーからの応答を受信するためのWebResponseを取得
            WebResponse webres = getWebResponse(url);

            // メモリに格納
            MemoryStream ms = new MemoryStream();

            //応答データを受信するためのStreamを取得
            using (Stream rs = webres.GetResponseStream()) {
                rs.ReadTimeout = 1000 * 60 * 5;
                byte[] buffer = new byte[65535];
                while (true) {
                    int rb = rs.Read(buffer, 0, buffer.Length);
                    if (rb > 0) {
                        ms.Write(buffer, 0, rb);
                    }
                    else {
                        break;
                    }
                }
            }

            // ファイルに保存
            FileStream fs = new FileStream(filePath, FileMode.Create);
            fs.Write(ms.GetBuffer(), 0, (int)ms.Length);
            fs.Close();
        }

        static WebResponse getWebResponse(string url) {
            WebResponse res = null;
            int retry = 0;
            WebException priorExp = null;
            do {
                try {
                    WebRequest req = WebRequest.Create(url);
                    req.Timeout = 1000 * 60 * 5;
                    res = req.GetResponse();
                }
                catch (WebException we) {
                    if (null != priorExp && !we.ToString().Equals(priorExp.ToString(), StringComparison.Ordinal))
                        retry = 0;
                    priorExp = we;
                    // とりあえず404以外の場合、6回繰り返す。
                    if (retry++ >= 6 ||
                        (we.Status == WebExceptionStatus.ProtocolError && ((HttpWebResponse)we.Response).StatusCode == HttpStatusCode.NotFound)) {
                        throw we;
                    }
                    Console.WriteLine("\n" + DateTime.Now.ToString() + "\n" + we.ToString());
                    Thread.Sleep(TimeSpan.FromSeconds(10));
                    Console.WriteLine("retry=" + retry);
                    continue;
                }
            } while (null == res);
            return res;
        }

    }
}
